{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train=pd.read_csv('./data/FE_train_LGBM.csv')\n",
    "test=pd.read_csv('./data/FE_test_LGBM.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "feats_to_encode=['msno','song_id']\n",
    "\n",
    "for col in feats_to_encode:\n",
    "    train[col] = le.fit_transform(train[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature1='song_length','song_year','genre_ids_counts'\n",
    "for aa in feature1:\n",
    "    median1=train[aa].median()\n",
    "    train[aa]=train[aa].replace(np.NaN,median1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del train['artist_name_counts']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cat_features = ['source_screen_name',\n",
    "       'source_system_tab', 'source_type','city', 'gender',\n",
    "       'registered_via', 'expiration_date_year', 'expiration_date_month',\n",
    "        'genre_ids', 'artist_name',\n",
    "       'language']\n",
    "train_cat=pd.get_dummies(train[cat_features])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train[cat_features]=train_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numerical_features=train[['bd','use_days','song_length','genre_ids_counts','song_id_counts','msno_counts']]\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "mms=MinMaxScaler()\n",
    "numerical_fetures=mms.fit_transform(numerical_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_num=pd.DataFrame(data=numerical_fetures,columns=['bd','use_days','song_length','genre_ids_counts','song_id_counts','msno_counts'])\n",
    "#train=pd.concat([train,train_num],axis=1)\n",
    "#train[numerical_fetures]=train_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for aa in numerical_features:\n",
    "   train[aa]=train_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_type</th>\n",
       "      <th>target</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>...</th>\n",
       "      <th>expiration_date_month</th>\n",
       "      <th>song_length</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>language</th>\n",
       "      <th>number_of_genres</th>\n",
       "      <th>song_year</th>\n",
       "      <th>genre_ids_counts</th>\n",
       "      <th>song_id_counts</th>\n",
       "      <th>msno_counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7602</td>\n",
       "      <td>173475</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>22</td>\n",
       "      <td>29245</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7602</td>\n",
       "      <td>93819</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>22</td>\n",
       "      <td>19505</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7602</td>\n",
       "      <td>18522</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>171</td>\n",
       "      <td>25096</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7602</td>\n",
       "      <td>147378</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>98</td>\n",
       "      <td>2690</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "      <td>0.22619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12443</td>\n",
       "      <td>7171</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0.25000</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0.25000</td>\n",
       "      <td>48</td>\n",
       "      <td>20446</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2007.0</td>\n",
       "      <td>0.25000</td>\n",
       "      <td>0.25000</td>\n",
       "      <td>0.25000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    msno  song_id  source_screen_name  source_system_tab  source_type  target  \\\n",
       "0   7602   173475                   8                  3            4     1.0   \n",
       "1   7602    93819                   8                  3            4     1.0   \n",
       "2   7602    18522                   8                  3            4     1.0   \n",
       "3   7602   147378                   8                  3            4     1.0   \n",
       "4  12443     7171                   8                  3            3     1.0   \n",
       "\n",
       "   city       bd  gender  registered_via     ...       expiration_date_month  \\\n",
       "0    11  0.22619       0               3     ...                           8   \n",
       "1    11  0.22619       0               3     ...                           8   \n",
       "2    11  0.22619       0               3     ...                           8   \n",
       "3    11  0.22619       0               3     ...                           8   \n",
       "4    13  0.25000       1               3     ...                           2   \n",
       "\n",
       "   song_length  genre_ids  artist_name  language  number_of_genres  song_year  \\\n",
       "0      0.22619         22        29245        10                 1     1999.0   \n",
       "1      0.22619         22        19505        10                 1     2006.0   \n",
       "2      0.22619        171        25096         1                 1     2010.0   \n",
       "3      0.22619         98         2690         7                 1     2014.0   \n",
       "4      0.25000         48        20446        10                 1     2007.0   \n",
       "\n",
       "   genre_ids_counts  song_id_counts  msno_counts  \n",
       "0           0.22619         0.22619      0.22619  \n",
       "1           0.22619         0.22619      0.22619  \n",
       "2           0.22619         0.22619      0.22619  \n",
       "3           0.22619         0.22619      0.22619  \n",
       "4           0.25000         0.25000      0.25000  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['target']\n",
    "X_train = train.drop([ \"target\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\sag.py:326: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l1', random_state=None, solver='saga', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression(penalty='l1',C=0.1,solver='saga')\n",
    "lr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "#y_test_pred_lr = lr.predict(test)\n",
    "y_train_pred_lr = lr.predict(X_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coef_log</th>\n",
       "      <th>coef_org</th>\n",
       "      <th>columns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.629599e-04</td>\n",
       "      <td>1.629599e-04</td>\n",
       "      <td>song_id_counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>6.256644e-07</td>\n",
       "      <td>6.256644e-07</td>\n",
       "      <td>artist_name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5.889068e-07</td>\n",
       "      <td>5.889068e-07</td>\n",
       "      <td>registration_init_time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>5.421935e-07</td>\n",
       "      <td>5.421935e-07</td>\n",
       "      <td>use_days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2.874357e-07</td>\n",
       "      <td>2.874357e-07</td>\n",
       "      <td>song_year</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.743495e-07</td>\n",
       "      <td>2.743495e-07</td>\n",
       "      <td>genre_ids</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3.170888e-08</td>\n",
       "      <td>3.170888e-08</td>\n",
       "      <td>genre_ids_counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.756687e-08</td>\n",
       "      <td>2.756687e-08</td>\n",
       "      <td>source_system_tab</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.234319e-08</td>\n",
       "      <td>2.234319e-08</td>\n",
       "      <td>expiration_date_month</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.930548e-08</td>\n",
       "      <td>1.930548e-08</td>\n",
       "      <td>song_id_counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.116354e-08</td>\n",
       "      <td>1.116354e-08</td>\n",
       "      <td>expiration_date_year</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4.540047e-09</td>\n",
       "      <td>4.540047e-09</td>\n",
       "      <td>registered_via</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.661757e-09</td>\n",
       "      <td>1.661757e-09</td>\n",
       "      <td>gender</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.520623e-09</td>\n",
       "      <td>1.520623e-09</td>\n",
       "      <td>song_length</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1.700134e-10</td>\n",
       "      <td>1.700134e-10</td>\n",
       "      <td>song_length</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1.013057e-10</td>\n",
       "      <td>1.013057e-10</td>\n",
       "      <td>use_days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4.803373e-18</td>\n",
       "      <td>4.803373e-18</td>\n",
       "      <td>genre_ids_counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>-2.522600e-11</td>\n",
       "      <td>-2.522600e-11</td>\n",
       "      <td>msno_counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>-4.344237e-10</td>\n",
       "      <td>-4.344237e-10</td>\n",
       "      <td>number_of_genres</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-3.591432e-09</td>\n",
       "      <td>-3.591432e-09</td>\n",
       "      <td>bd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.430130e-08</td>\n",
       "      <td>-1.430130e-08</td>\n",
       "      <td>city</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-4.064799e-08</td>\n",
       "      <td>-4.064799e-08</td>\n",
       "      <td>language</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-4.758558e-08</td>\n",
       "      <td>-4.758558e-08</td>\n",
       "      <td>song_id</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-2.373766e-07</td>\n",
       "      <td>-2.373766e-07</td>\n",
       "      <td>msno_counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-3.021122e-07</td>\n",
       "      <td>-3.021122e-07</td>\n",
       "      <td>bd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-3.187633e-07</td>\n",
       "      <td>-3.187633e-07</td>\n",
       "      <td>source_type</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-3.648428e-07</td>\n",
       "      <td>-3.648428e-07</td>\n",
       "      <td>source_screen_name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-5.970112e-07</td>\n",
       "      <td>-5.970112e-07</td>\n",
       "      <td>expiration_date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-2.101754e-06</td>\n",
       "      <td>-2.101754e-06</td>\n",
       "      <td>msno</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        coef_log      coef_org                 columns\n",
       "21  1.629599e-04  1.629599e-04          song_id_counts\n",
       "16  6.256644e-07  6.256644e-07             artist_name\n",
       "9   5.889068e-07  5.889068e-07  registration_init_time\n",
       "11  5.421935e-07  5.421935e-07                use_days\n",
       "19  2.874357e-07  2.874357e-07               song_year\n",
       "15  2.743495e-07  2.743495e-07               genre_ids\n",
       "20  3.170888e-08  3.170888e-08        genre_ids_counts\n",
       "3   2.756687e-08  2.756687e-08       source_system_tab\n",
       "13  2.234319e-08  2.234319e-08   expiration_date_month\n",
       "27  1.930548e-08  1.930548e-08          song_id_counts\n",
       "12  1.116354e-08  1.116354e-08    expiration_date_year\n",
       "8   4.540047e-09  4.540047e-09          registered_via\n",
       "7   1.661757e-09  1.661757e-09                  gender\n",
       "14  1.520623e-09  1.520623e-09             song_length\n",
       "25  1.700134e-10  1.700134e-10             song_length\n",
       "24  1.013057e-10  1.013057e-10                use_days\n",
       "26  4.803373e-18  4.803373e-18        genre_ids_counts\n",
       "28 -2.522600e-11 -2.522600e-11             msno_counts\n",
       "18 -4.344237e-10 -4.344237e-10        number_of_genres\n",
       "23 -3.591432e-09 -3.591432e-09                      bd\n",
       "5  -1.430130e-08 -1.430130e-08                    city\n",
       "17 -4.064799e-08 -4.064799e-08                language\n",
       "1  -4.758558e-08 -4.758558e-08                 song_id\n",
       "22 -2.373766e-07 -2.373766e-07             msno_counts\n",
       "6  -3.021122e-07 -3.021122e-07                      bd\n",
       "4  -3.187633e-07 -3.187633e-07             source_type\n",
       "2  -3.648428e-07 -3.648428e-07      source_screen_name\n",
       "10 -5.970112e-07 -5.970112e-07         expiration_date\n",
       "0  -2.101754e-06 -2.101754e-06                    msno"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_names = X_train.columns\n",
    "fs = pd.DataFrame({\"columns\":list(feat_names), \"coef_org\":list((lr.coef_[0,:].T)),\"coef_log\":list((lr.coef_[0,:].T))})\n",
    "fs.sort_values(by=['coef_org'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6693933470145252\n",
      "{'C': 0.1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "#正则化的 Logistic Regression\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=555)\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1,0.5, 1]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')#Logistic Regression + GridSearchCV\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=3, scoring='neg_log_loss',n_jobs = 4,)\n",
    "grid.fit(X_train,y_train)\n",
    "\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_best = grid.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy = cross_val_score(lr_best, X_train, y_train, cv=3, scoring='accuracy')\n",
    "print ('accuracy of each fold is: ',accuracy)#accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "from sklearn.externals import joblib\n",
    "#from sklearn.preprocessing import  OneHotEncoder\n",
    "\n",
    "from scipy.sparse import hstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.3, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bynode=1, colsample_bytree=1, gamma=0, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=15, min_child_weight=5, missing=None,\n",
       "       n_estimators=300, n_jobs=1, nthread=None,\n",
       "       objective='binary:logistic', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=None,\n",
       "       subsample=1, verbosity=1)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = xgb.XGBClassifier(learning_rate=0.1, max_depth=15, min_child_weight=5, n_estimators=300)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[458127 191888]\n",
      " [165988 512745]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.73      0.70      0.72    650015\n",
      "        1.0       0.73      0.76      0.74    678733\n",
      "\n",
      "avg / total       0.73      0.73      0.73   1328748\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix,classification_report\n",
    "y_pred=model.predict(X_test)\n",
    "print(confusion_matrix(y_test,y_pred))\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ny_pred_test = model.predict_proba(X_test.values)\\nxgb_test_auc = roc_auc_score(y_test, y_pred_test)\\nprint('xgboost test auc: %.5f' % xgb_test_auc)\\n\""
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "y_pred_test = model.predict_proba(X_test.values)\n",
    "xgb_test_auc = roc_auc_score(y_test, y_pred_test)\n",
    "print('xgboost test auc: %.5f' % xgb_test_auc)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle as cPickle\n",
    "\n",
    "cPickle.dump(model, open(\"XGBoost_org.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "feats_to_encode=['msno','song_id']\n",
    "\n",
    "for col in feats_to_encode:\n",
    "    test[col] = le.fit_transform(test[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature1='song_length','song_year','genre_ids_counts'\n",
    "for aa in feature1:\n",
    "    median2=test[aa].median()\n",
    "    test[aa]=train[aa].replace(np.NaN,median2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#del test['artist_name_counts']\n",
    "cat_features = ['source_screen_name',\n",
    "       'source_system_tab', 'source_type','city', 'gender',\n",
    "       'registered_via', 'expiration_date_year', 'expiration_date_month',\n",
    "        'genre_ids', 'artist_name',\n",
    "       'language']\n",
    "test_cat=pd.get_dummies(test[cat_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test[cat_features]=test_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = cPickle.load(open(\"XGBoost_org.pkl\", 'rb'))\n",
    "\n",
    "#输出每类的概率\n",
    "y_test_pred = model.predict_proba(test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1510243, 23)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nxgboost = model\\nX_train_leaves = xgboost.apply(X_train.values)\\nX_test_leaves = xgboost.apply(X_test.values)\\n\\n# 训练样本个数\\ntrain_rows = X_train_leaves.shape[0]\\n# 合并编码后的训练数据和测试数据\\nX_leaves = np.concatenate((X_train_leaves, X_test_leaves), axis=0)\\nX_leaves = X_leaves.astype(np.int32)\\n(rows, cols) = X_leaves.shape\\n\\n# 定义LR模型\\nfrom sklearn.linear_model import LogisticRegression\\nlr = LogisticRegression(penalty='l1',C=0.1,solver='saga')\\n# lr对xgboost特征编码后的样本模型训练\\nlr.fit(X_trans[:train_rows, :], y_train)\\ny_pred_xgblr1 = lr.predict_proba(X_trans[train_rows:, :])\\n# 预测及AUC评测\\n#y_pred_xgblr1 = lr.predict_proba(X_trans[train_rows:, :])[:, 1]\\n# y_pred_xgblr1.shape  = (112,)\\nxgb_lr_auc1 = roc_auc_score(pd.get_dummies(y_test), y_pred_xgblr1)\\nprint('基于Xgb特征编码后的LR AUC: %.5f' % xgb_lr_auc1)\\n\""
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "xgboost = model\n",
    "X_train_leaves = xgboost.apply(X_train.values)\n",
    "X_test_leaves = xgboost.apply(X_test.values)\n",
    "\n",
    "# 训练样本个数\n",
    "train_rows = X_train_leaves.shape[0]\n",
    "# 合并编码后的训练数据和测试数据\n",
    "X_leaves = np.concatenate((X_train_leaves, X_test_leaves), axis=0)\n",
    "X_leaves = X_leaves.astype(np.int32)\n",
    "(rows, cols) = X_leaves.shape\n",
    "\n",
    "# 定义LR模型\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression(penalty='l1',C=0.1,solver='saga')\n",
    "# lr对xgboost特征编码后的样本模型训练\n",
    "lr.fit(X_trans[:train_rows, :], y_train)\n",
    "y_pred_xgblr1 = lr.predict_proba(X_trans[train_rows:, :])\n",
    "# 预测及AUC评测\n",
    "#y_pred_xgblr1 = lr.predict_proba(X_trans[train_rows:, :])[:, 1]\n",
    "# y_pred_xgblr1.shape  = (112,)\n",
    "xgb_lr_auc1 = roc_auc_score(pd.get_dummies(y_test), y_pred_xgblr1)\n",
    "print('基于Xgb特征编码后的LR AUC: %.5f' % xgb_lr_auc1)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
